require 'rubygems'
require 'hpricot'
require 'open-uri'
require 'pdf/reader'

class PageTextReceiver
    attr_accessor :content

    def initialize
      @content = []
    end

    # Called when page parsing starts
    def begin_page(arg = nil)
      @content << ""
    end

    # record text that is drawn on the page
    def show_text(string, *params)
      @content.last << string.strip
    end

    # there's a few text callbacks, so make sure we process them all
    alias :super_show_text :show_text
    alias :move_to_next_line_and_show_text :show_text
    alias :set_spacing_next_line_show_text :show_text

    # this final text callback takes slightly different arguments
    def show_text_with_positioning(*params)
      params = params.first
      params.each { |str| show_text(str) if str.kind_of?(String)}
    end
end

#rake -T will get you a list of available commands.
#most in here are scrapers relating to db:sync

#Probably something easier than this in ruby, 
# but from : http://snippets.dzone.com/posts/show/302
class Array
  def to_hash_keys(&block)
    Hash[*self.collect { |v|
      [v, block.call(v)]
    }.flatten]
  end

  def to_hash_values(&block)
    Hash[*self.collect { |v|
      [block.call(v), v]
    }.flatten]
  end
end

namespace "db:sync" do
  desc "Populates an empty database with the default required configuration database, or updates a pre-existing database. "
  task :all => [ :environment, "db:sync:wards", "db:sync:parties","db:sync:councillors","db:sync:committees", "db:sync:meetings"] do
  end
  
  desc "Populates an empty database with the party names registered on the site."
  task :parties => [:environment] do
    puts '* Populating Parties'
    ward_list_url = "#{MCC_SITE_SCRIPTS_URL}/councillors.php?viewBy=party"
    puts "Opening"+ward_list_url

    #Open the link and tidy it up with Hpricot
    doc = Hpricot( open(ward_list_url) )

    #Pull out the list of results
    list = doc.search("//ul[@class='list']/li/a")

    puts "We have "+list.size.to_s
    href=""
    list.each do |entry|
      name = entry.inner_text
      puts "-Party:"+name
      party = Party.new
      party.name = name
      check_exists = Party.find(:first, :conditions => ["name=?",name] )
      if check_exists.nil?
        party.save
        puts "save "
      else
        puts "duplicate"
      end
      puts "---------"
    end
  end

  desc "Populates an empty database with the wards registered on the site."
  task :wards => [:environment] do
    puts '* Populating Wards'
    ward_list_url = "#{MCC_SITE_SCRIPTS_URL}/councillors.php?viewBy=ward"
    doc = Hpricot( open(ward_list_url) )
    puts "Opening"+ward_list_url
    list = doc.search("//ul[@class='list']/li/a")

    puts "we have "+list.size.to_s
    href=""
    list.each do |entry|
      name = entry.inner_text
      puts "-Ward:"+name

      ward = Ward.new
      ward.name = name
      check_exists = Ward.find(:first, :conditions => ["name=?",name] )
      if check_exists.nil?
        ward.save
        puts "save "
      else
        puts "duplicate"
      end
      puts "---------"
    end
  end
  
  desc "Populates an empty database with the councillors registered on the site, or updates a pre-existing database. "
 # task :councillors => [:environment, "db:sync:wards", "db:sync:parties"] do
  task :councillors => [:environment] do  # 1
    
    parties= Party.find(:all)
    wards= Ward.find(:all)
    
    party_hashes= parties.to_hash_values {|v| v.name}
    ward_hashes= wards.to_hash_values {|v| v.name}
    
    
    puts '* Populating Councillors'
    ward_list_url = "#{MCC_SITE_SCRIPTS_URL}/councillors.php?viewBy=name"
    doc = Hpricot( open(ward_list_url) )
    puts "Opening " + ward_list_url
    list = doc.search("//dl[@class='councillor_box']/dt")

    
    i = 0
    j = 0
    href=""
    new_councillors = 0
    duplicate_councillors = 0
    puts list.size.to_s + " councillors found, importing: "
    list.each do |entry| # 2
      link = entry.at('a') # <a> inside the <dt
      if link.nil? # 3
        i = i + 1
        print "x"
        $stdout.flush
      else
        j = j + 1
        href = link['href']
      #Open the Biog page to strip more info out.
      councillor_doc = Hpricot(open(href))
      biog= councillor_doc.search("#coucillor_biog")
      #Pull out the sub-sections
      chunks= biog.search('p')
      ward= chunks[0].inner_text.sub(/Ward:\s*/,"").strip
      party= chunks[1].inner_text.strip.sub(/\s*Party$/,"").strip
      address= Address.construct_from_html( biog.search('address/span').inner_html )
      email_chunk= chunks.at('.email')
      email= email_chunk.inner_text unless email_chunk.nil?
      end_of_term= nil
      fax= nil
      chunks.each do |p| #4
        if p.inner_text =~ /End of Term.+/ #5
            end_of_term = p.inner_text.sub(/End of Term\s*/,'').strip
        end #5
        if p.inner_text =~ /Fax:.+/ #6
          fax= p.inner_text.sub(/Fax:\s*/,'').strip
        end #6
      end#4      
      name = link.inner_text
      name_parts = name.split(" ")
      home_telephone= nil
      home_mobile= nil
      town_hall_telephone= nil
      telephone_chunk= chunks.at('.tel')
      telephone= telephone_chunk.inner_text unless telephone_chunk.nil?
      unless telephone.nil? #7
        home_mobile = telephone if telephone =~ /07.+/
        home_telephone = telephone if telephone =~ /01.+/
      # otherwise we need to figure it out as the format is a bit odd (Home) xxxxx (Mobile) yyyy
      end     #7
      councillor = Councillor.new
      councillor.first_name=name_parts[0]
      councillor.last_name=name_parts[1]
      councillor.official_council_webpage_listing=href
      councillor.party= party_hashes[party]
      councillor.ward= ward_hashes[ward]
      councillor.address= address
      councillor.email_address= email
      councillor.end_of_term= end_of_term
      councillor.fax= fax
      councillor.home_telephone= home_telephone
      councillor.home_mobile= home_mobile
      
#      t.string   "common_abbreviation"
#      t.string   "town_hall_telephone"
#      t.text     "further_information"
      
      
      check_exists = Councillor.find(:first, :conditions => ["official_council_webpage_listing=?",councillor.official_council_webpage_listing] )
      if check_exists.nil? #8
        councillor.save
        new_councillors += 1
      else 
        duplicate_councillors += 1
      end #8
      print "X"
      $stdout.flush
    end #3  
  end #2
  puts new_councillors.to_s + " new councillors saved"
    puts duplicate_councillors.to_s + " duplicates found"
    puts 'not found : ' + i.to_s + ' found ' + j.to_s
end #1
  desc "Updates the list of committees"
  task :committees => [:environment] do
    puts '* Populating Committees'
    ward_list_url = "#{MCC_SITE_SCRIPTS_URL}/meetings_index.php"
    doc = Hpricot( open(ward_list_url) )
    puts "Opening " + ward_list_url
    list = doc.search("//div[@class='display_box']/ul/li/a")
    puts list.size.to_s + " committees"
    duplicate_committees = 0
    new_committees = 0
    debugger
    list.each do |committee|
      committee_name= Committee.stem_name( committee.inner_html.to_s )
      if Committee.find_by_name(committee_name)
        duplicate_committees += 1
      else
        Committee.new(:name => committee_name).save
        new_committees += 1
      end
    end
    puts new_committees.to_s + " new committees saved"
    puts duplicate_committees.to_s + " duplicates found"
  end

  desc "Updates the list of meetings of committees"
  task :meetings => [:environment] do
    puts '* Populating Meetings'
    committee_list_url = "#{MCC_SITE_SCRIPTS_URL}/meetings_index.php"
    doc = Hpricot( open(committee_list_url) )
    puts "Opening " + committee_list_url
    committees = doc.search("//div[@class='display_box']/ul/li/a")
    puts committees.size.to_s + " committees"
    new_meetings = 0
    duplicate_meetings = 0
    committees.each do |committee|
      @committee = Committee.find_by_name( Committee.stem_name(committee.inner_html) )
      unless @committee.nil?
          puts "processing "+@committee.name
          meeting_page = Hpricot(open(committee['href']))
          meeting_list = meeting_page.search("//ul[@class='list']/li/a")
          meeting_list.each do |meeting|          
            @meeting =Meeting.find(:first, :conditions => {:committee_id => @committee.id, :date => Date.parse(meeting.inner_html)})
            if @meeting
              puts ":Duplicate:"
              duplicate_meetings += 1
            else
              puts ":Saving:"
              @meeting = Meeting.new(:committee_id => @committee.id, :date => Date.parse(meeting.inner_html), :website_link => meeting['href'])
              @meeting.save!
              new_meetings += 1
            end
           puts @meeting.website_link 
           meeting_page = Hpricot(open(@meeting.website_link))  
           meeting_links = meeting_page.search("//div[@class='display_box meeting_attach']/p/a")
           meeting_links.each do |m|
             if m.inner_html =~ /(Download|Read) the Minutes/
               puts m.inner_html
               puts m['href']
               @meeting.update_attributes(:minutes_pdf_link => m['href'])
             end
           end
         end
       else
           puts "Missing committee: " + committee.inner_html
        end
    end
    puts new_meetings.to_s + " new meetings saved."
    puts duplicate_meetings.to_s + " duplicate meetings skipped."

  end

  desc "Update the list of attendees for meetings"
  task :attendees => [:environment] do
    meetings = Meeting.find(:first, :conditions => ["minutes_pdf_link <> ''"])
    puts meetings.minutes_pdf_link
    
  end
  
end